@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eedac9dba0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

fix: revert code to avoid no attribute problem (#827)

Wen Sun committed Aug 23, 2023 at 02:55 UTC eedac9dba0e586820a797685dab26807439a3c2c
1 file changed +4 -3
vllm/model_executor/models/gpt_bigcode.py
+4 -3
@@ -259,6 +259,8 @@ class GPTBigCodeForCausalLM(nn.Module):
259 model_name_or_path: str,
260 cache_dir: Optional[str] = None,
261 use_np_cache: bool = False):
262 + tensor_model_parallel_world_size = (
263 + get_tensor_model_parallel_world_size())
264 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
265 state_dict = self.state_dict()
266
@@ -288,8 +290,7 @@ class GPTBigCodeForCausalLM(nn.Module):
290 hidden_size = self.config.hidden_size
291 head_size = hidden_size // total_num_heads
292 total_kv_size = head_size * total_num_kv_heads
291 - num_heads = (total_num_heads //
292 - self.tensor_model_parallel_world_size)
293 + num_heads = total_num_heads // tensor_model_parallel_world_size
294 head_start = tensor_model_parallel_rank * num_heads
295 head_end = (tensor_model_parallel_rank + 1) * num_heads
296
@@ -329,7 +330,7 @@ class GPTBigCodeForCausalLM(nn.Module):
330 if name == "transformer.wte.weight":
331 # Consider padding in the vocab size.
332 padded_vocab_size = param.shape[
332 - 0] * self.tensor_model_parallel_world_size
333 + 0] * tensor_model_parallel_world_size
334 num_extra_rows = padded_vocab_size - self.config.vocab_size
335 extra_rows = torch.empty(num_extra_rows,
336 loaded_weight.shape[1])