A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[BugFix][TritonMLA] Process weights after model loading for GGUF (#14555)
Signed-off-by: TianyuanWu <Tianyuan.Wu@amd.com>
TY-AMD committed
Mar 13, 2025 at 11:14 UTC
128bf7528370d792099c66f301c6c5deef8f4110
1 file changed
+4
-1
vllm/model_executor/model_loader/loader.py
+4
-1
@@ -1330,11 +1330,14 @@ class GGUFModelLoader(BaseModelLoader):
1330
local_model_path, gguf_weights_map):
1331
model_config.hf_config.update({"tie_word_embeddings": True})
1332
1333
+ target_device = torch.device(device_config.device)
1334
with set_default_torch_dtype(model_config.dtype):
1334
- with torch.device(device_config.device):
1335
+ with target_device:
1336
model = _initialize_model(vllm_config=vllm_config)
1337
model.load_weights(
1338
self._get_weights_iterator(local_model_path, gguf_weights_map))
1339
+
1340
+ _process_weights_after_loading(model, model_config, target_device)
1341
return model
1342
1343