@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8334c39f37
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix new Llama3.1 GGUF model loading (#7269)

Isotr0py committed Aug 9, 2024 at 04:42 UTC 8334c39f373b787a20eff8b7655363dd764dfe57
1 file changed +16 -14
vllm/model_executor/model_loader/weight_utils.py
+16 -14
@@ -435,23 +435,25 @@ def gguf_quant_weights_iterator(
435 reader = gguf.GGUFReader(gguf_file)
436
437 for tensor in reader.tensors:
438 - weight_type = tensor.tensor_type
439 - name = gguf_to_hf_name_map[tensor.name]
438 + if tensor.name in gguf_to_hf_name_map:
439 + weight_type = tensor.tensor_type
440 + name = gguf_to_hf_name_map[tensor.name]
441
441 - if weight_type.name != "F32":
442 - weight_type_name = name.replace("weight", "qweight_type")
443 - weight_type = torch.tensor(weight_type)
444 - yield weight_type_name, weight_type
442 + if weight_type.name != "F32":
443 + weight_type_name = name.replace("weight", "qweight_type")
444 + weight_type = torch.tensor(weight_type)
445 + yield weight_type_name, weight_type
446
447 for tensor in reader.tensors:
447 - weight = tensor.data
448 - weight_type = tensor.tensor_type
449 - name = gguf_to_hf_name_map[tensor.name]
450 -
451 - if weight_type.name != "F32":
452 - name = name.replace("weight", "qweight")
453 - param = torch.tensor(weight)
454 - yield name, param
448 + if tensor.name in gguf_to_hf_name_map:
449 + weight = tensor.data
450 + weight_type = tensor.tensor_type
451 + name = gguf_to_hf_name_map[tensor.name]
452 +
453 + if weight_type.name != "F32":
454 + name = name.replace("weight", "qweight")
455 + param = torch.tensor(weight)
456 + yield name, param
457
458
459 def kv_cache_scales_loader(