@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4f419c00a6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

Fix ShardedStateLoader for vllm fp8 quantization (#7708)

Flex Wang committed Aug 22, 2024 at 05:25 UTC 4f419c00a621eac24f954f2b7670cbd22eb232a8
1 file changed +4
vllm/model_executor/model_loader/loader.py
+4
@@ -579,6 +579,10 @@ class ShardedStateLoader(BaseModelLoader):
579 with torch.device(device_config.device):
580 model = _initialize_model(model_config, self.load_config,
581 lora_config, cache_config)
582 + for _, module in model.named_modules():
583 + quant_method = getattr(module, "quant_method", None)
584 + if quant_method is not None:
585 + quant_method.process_weights_after_loading(module)
586 rank = get_tensor_model_parallel_rank()
587 pattern = os.path.join(
588 local_model_path,