A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
fix: revert code to avoid no attribute problem (#827)
Wen Sun committed
Aug 23, 2023 at 02:55 UTC
eedac9dba0e586820a797685dab26807439a3c2c
1 file changed
+4
-3
vllm/model_executor/models/gpt_bigcode.py
+4
-3
@@ -259,6 +259,8 @@ class GPTBigCodeForCausalLM(nn.Module):
259
model_name_or_path: str,
260
cache_dir: Optional[str] = None,
261
use_np_cache: bool = False):
262
+ tensor_model_parallel_world_size = (
263
+ get_tensor_model_parallel_world_size())
264
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
265
state_dict = self.state_dict()
266
@@ -288,8 +290,7 @@ class GPTBigCodeForCausalLM(nn.Module):
290
hidden_size = self.config.hidden_size
291
head_size = hidden_size // total_num_heads
292
total_kv_size = head_size * total_num_kv_heads
291
- num_heads = (total_num_heads //
292
- self.tensor_model_parallel_world_size)
293
+ num_heads = total_num_heads // tensor_model_parallel_world_size
294
head_start = tensor_model_parallel_rank * num_heads
295
head_end = (tensor_model_parallel_rank + 1) * num_heads
296
@@ -329,7 +330,7 @@ class GPTBigCodeForCausalLM(nn.Module):
330
if name == "transformer.wte.weight":
331
# Consider padding in the vocab size.
332
padded_vocab_size = param.shape[
332
- 0] * self.tensor_model_parallel_world_size
333
+ 0] * tensor_model_parallel_world_size
334
num_extra_rows = padded_vocab_size - self.config.vocab_size
335
extra_rows = torch.empty(num_extra_rows,
336
loaded_weight.shape[1])