@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9f6be8692e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix config for Falcon (#1164)

Woosuk Kwon committed Sep 23, 2023 at 17:38 UTC 9f6be8692e09b0ea706ecbb6693c54e210a967f9
2 files changed +8 -3
vllm/config.py
+6 -1
@@ -135,7 +135,8 @@ class ModelConfig:
135 # FIXME(woosuk): This may not be true for all models.
136 return self.hf_config.hidden_size // self.hf_config.num_attention_heads
137
138 - def get_num_heads(self, parallel_config: "ParallelConfig") -> int:
138 + def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
139 + """Returns the number of KV heads per GPU worker."""
140 # For GPTBigCode & Falcon:
141 # Note: for falcon, when new_decoder_architecture is True, the
142 # multi_query flag is ignored and we use n_head_kv for the number of
@@ -147,11 +148,15 @@ class ModelConfig:
148 if not new_decoder_arch_falcon and getattr(self.hf_config,
149 "multi_query", False):
150 # Multi-query attention, only one KV head.
151 + # Currently, tensor parallelism is not supported in this case.
152 return 1
153 # For Falcon:
154 if getattr(self.hf_config, "n_head_kv", None) is not None:
155 return (self.hf_config.n_head_kv //
156 parallel_config.tensor_parallel_size)
157 + if getattr(self.hf_config, "num_kv_heads", None) is not None:
158 + return (self.hf_config.num_kv_heads //
159 + parallel_config.tensor_parallel_size)
160 # For LLaMA-2:
161 if getattr(self.hf_config, "num_key_value_heads", None) is not None:
162 return (self.hf_config.num_key_value_heads //
vllm/worker/cache_engine.py
+2 -2
@@ -33,7 +33,7 @@ class CacheEngine:
33
34 self.head_size = model_config.get_head_size()
35 self.num_layers = model_config.get_num_layers(parallel_config)
36 - self.num_heads = model_config.get_num_heads(parallel_config)
36 + self.num_heads = model_config.get_num_kv_heads(parallel_config)
37 self.dtype = model_config.dtype
38
39 self.block_size = cache_config.block_size
@@ -146,7 +146,7 @@ class CacheEngine:
146 parallel_config: ParallelConfig,
147 ) -> int:
148 head_size = model_config.get_head_size()
149 - num_heads = model_config.get_num_heads(parallel_config)
149 + num_heads = model_config.get_num_kv_heads(parallel_config)
150 num_layers = model_config.get_num_layers(parallel_config)
151
152 key_cache_block = block_size * num_heads * head_size