A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Fix new Llama3.1 GGUF model loading (#7269)
Isotr0py committed
Aug 9, 2024 at 04:42 UTC
8334c39f373b787a20eff8b7655363dd764dfe57
1 file changed
+16
-14
vllm/model_executor/model_loader/weight_utils.py
+16
-14
@@ -435,23 +435,25 @@ def gguf_quant_weights_iterator(
435
reader = gguf.GGUFReader(gguf_file)
436
437
for tensor in reader.tensors:
438
- weight_type = tensor.tensor_type
439
- name = gguf_to_hf_name_map[tensor.name]
438
+ if tensor.name in gguf_to_hf_name_map:
439
+ weight_type = tensor.tensor_type
440
+ name = gguf_to_hf_name_map[tensor.name]
441
441
- if weight_type.name != "F32":
442
- weight_type_name = name.replace("weight", "qweight_type")
443
- weight_type = torch.tensor(weight_type)
444
- yield weight_type_name, weight_type
442
+ if weight_type.name != "F32":
443
+ weight_type_name = name.replace("weight", "qweight_type")
444
+ weight_type = torch.tensor(weight_type)
445
+ yield weight_type_name, weight_type
446
447
for tensor in reader.tensors:
447
- weight = tensor.data
448
- weight_type = tensor.tensor_type
449
- name = gguf_to_hf_name_map[tensor.name]
450
-
451
- if weight_type.name != "F32":
452
- name = name.replace("weight", "qweight")
453
- param = torch.tensor(weight)
454
- yield name, param
448
+ if tensor.name in gguf_to_hf_name_map:
449
+ weight = tensor.data
450
+ weight_type = tensor.tensor_type
451
+ name = gguf_to_hf_name_map[tensor.name]
452
+
453
+ if weight_type.name != "F32":
454
+ name = name.replace("weight", "qweight")
455
+ param = torch.tensor(weight)
456
+ yield name, param
457
458
459
def kv_cache_scales_loader(