A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
add vocab padding for LLama(Support WizardLM) (#411)
panda committed
Jul 14, 2023 at 11:56 UTC
7b6ae94059a7c7f097f32987489fb03b02cbf613
1 file changed
+17
-4
vllm/model_executor/models/llama.py
+17
-4
@@ -187,10 +187,9 @@ class LlamaModel(nn.Module):
187
self.padding_idx = config.pad_token_id
188
self.vocab_size = config.vocab_size
189
190
+ vocab_size = ((config.vocab_size + 63) // 64) * 64
191
self.embed_tokens = VocabParallelEmbedding(
191
- config.vocab_size,
192
- config.hidden_size,
193
- perform_initialization=False)
192
+ vocab_size, config.hidden_size, perform_initialization=False)
193
self.layers = nn.ModuleList([
194
LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)
195
])
@@ -228,8 +227,9 @@ class LlamaForCausalLM(nn.Module):
227
super().__init__()
228
self.config = config
229
self.model = LlamaModel(config)
230
+ vocab_size = ((config.vocab_size + 63) // 64) * 64
231
self.lm_head = ColumnParallelLinear(config.hidden_size,
232
- config.vocab_size,
232
+ vocab_size,
233
bias=False,
234
gather_output=False,
235
perform_initialization=False)
@@ -259,6 +259,8 @@ class LlamaForCausalLM(nn.Module):
259
model_name_or_path: str,
260
cache_dir: Optional[str] = None,
261
use_np_cache: bool = False):
262
+ tensor_model_parallel_world_size = (
263
+ get_tensor_model_parallel_world_size())
264
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
265
state_dict = self.state_dict()
266
@@ -267,6 +269,17 @@ class LlamaForCausalLM(nn.Module):
269
if "rotary_emb.inv_freq" in name:
270
continue
271
272
+ if "embed_tokens" in name or "lm_head" in name:
273
+ param = state_dict[name]
274
+ # Consider padding in the vocab size.
275
+ padded_vocab_size = (param.shape[0] *
276
+ tensor_model_parallel_world_size)
277
+ num_extra_rows = padded_vocab_size - self.config.vocab_size
278
+ extra_rows = torch.empty(num_extra_rows,
279
+ loaded_weight.shape[1])
280
+ extra_rows = extra_rows.to(loaded_weight)
281
+ loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
282
+
283
is_attention_weight = False
284
for stride_id, att_weight_name in enumerate(
285
["q_proj", "k_proj", "v_proj"]):