A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
use get_tensor in safe_open (#1696)
twaka committed
Nov 19, 2023 at 09:45 UTC
e946260cf3c7a9d5a463a1babae4cebb433f7ffd
1 file changed
+3
-8
vllm/model_executor/weight_utils.py
+3
-8
@@ -243,8 +243,8 @@ def hf_model_weights_iterator(
243
for st_file in hf_weights_files:
244
with safe_open(st_file, framework="pt") as f:
245
for name in f.keys():
246
- param = f.get_slice(name)
247
- yield name, convert_pyslice_to_tensor(param)
246
+ param = f.get_tensor(name)
247
+ yield name, param
248
else:
249
for bin_file in hf_weights_files:
250
state = torch.load(bin_file, map_location="cpu")
@@ -265,12 +265,7 @@ def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
265
tensor first.
266
"""
267
if not isinstance(x, torch.Tensor):
268
- try:
269
- x = x[:]
270
- except IndexError:
271
- # IndexError happens when the tensor is empty.
272
- # transformer.h.0.attn.masked_bias is empty in some gpt2 models.
273
- return torch.Tensor()
268
+ x = x[:]
269
return x
270
271