@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7b6ae94059
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

add vocab padding for LLama(Support WizardLM) (#411)

panda committed Jul 14, 2023 at 11:56 UTC 7b6ae94059a7c7f097f32987489fb03b02cbf613
1 file changed +17 -4
vllm/model_executor/models/llama.py
+17 -4
@@ -187,10 +187,9 @@ class LlamaModel(nn.Module):
187 self.padding_idx = config.pad_token_id
188 self.vocab_size = config.vocab_size
189
190 + vocab_size = ((config.vocab_size + 63) // 64) * 64
191 self.embed_tokens = VocabParallelEmbedding(
191 - config.vocab_size,
192 - config.hidden_size,
193 - perform_initialization=False)
192 + vocab_size, config.hidden_size, perform_initialization=False)
193 self.layers = nn.ModuleList([
194 LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)
195 ])
@@ -228,8 +227,9 @@ class LlamaForCausalLM(nn.Module):
227 super().__init__()
228 self.config = config
229 self.model = LlamaModel(config)
230 + vocab_size = ((config.vocab_size + 63) // 64) * 64
231 self.lm_head = ColumnParallelLinear(config.hidden_size,
232 - config.vocab_size,
232 + vocab_size,
233 bias=False,
234 gather_output=False,
235 perform_initialization=False)
@@ -259,6 +259,8 @@ class LlamaForCausalLM(nn.Module):
259 model_name_or_path: str,
260 cache_dir: Optional[str] = None,
261 use_np_cache: bool = False):
262 + tensor_model_parallel_world_size = (
263 + get_tensor_model_parallel_world_size())
264 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
265 state_dict = self.state_dict()
266
@@ -267,6 +269,17 @@ class LlamaForCausalLM(nn.Module):
269 if "rotary_emb.inv_freq" in name:
270 continue
271
272 + if "embed_tokens" in name or "lm_head" in name:
273 + param = state_dict[name]
274 + # Consider padding in the vocab size.
275 + padded_vocab_size = (param.shape[0] *
276 + tensor_model_parallel_world_size)
277 + num_extra_rows = padded_vocab_size - self.config.vocab_size
278 + extra_rows = torch.empty(num_extra_rows,
279 + loaded_weight.shape[1])
280 + extra_rows = extra_rows.to(loaded_weight)
281 + loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
282 +
283 is_attention_weight = False
284 for stride_id, att_weight_name in enumerate(
285 ["q_proj", "k_proj", "v_proj"]):