@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e946260cf3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

use get_tensor in safe_open (#1696)

twaka committed Nov 19, 2023 at 09:45 UTC e946260cf3c7a9d5a463a1babae4cebb433f7ffd
1 file changed +3 -8
vllm/model_executor/weight_utils.py
+3 -8
@@ -243,8 +243,8 @@ def hf_model_weights_iterator(
243 for st_file in hf_weights_files:
244 with safe_open(st_file, framework="pt") as f:
245 for name in f.keys():
246 - param = f.get_slice(name)
247 - yield name, convert_pyslice_to_tensor(param)
246 + param = f.get_tensor(name)
247 + yield name, param
248 else:
249 for bin_file in hf_weights_files:
250 state = torch.load(bin_file, map_location="cpu")
@@ -265,12 +265,7 @@ def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
265 tensor first.
266 """
267 if not isinstance(x, torch.Tensor):
268 - try:
269 - x = x[:]
270 - except IndexError:
271 - # IndexError happens when the tensor is empty.
272 - # transformer.h.0.attn.masked_bias is empty in some gpt2 models.
273 - return torch.Tensor()
268 + x = x[:]
269 return x
270
271