A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
Fix ShardedStateLoader for vllm fp8 quantization (#7708)
Flex Wang committed
Aug 22, 2024 at 05:25 UTC
4f419c00a621eac24f954f2b7670cbd22eb232a8
1 file changed
+4
vllm/model_executor/model_loader/loader.py
+4
@@ -579,6 +579,10 @@ class ShardedStateLoader(BaseModelLoader):
579
with torch.device(device_config.device):
580
model = _initialize_model(model_config, self.load_config,
581
lora_config, cache_config)
582
+ for _, module in model.named_modules():
583
+ quant_method = getattr(module, "quant_method", None)
584
+ if quant_method is not None:
585
+ quant_method.process_weights_after_loading(module)
586
rank = get_tensor_model_parallel_rank()
587
pattern = os.path.join(
588
local_model_path,