A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Misc] add process_weights_after_loading for DummyLoader (#8969)
Divakar Verma committed
Sep 30, 2024 at 22:46 UTC
82f3937e599a4f088a62e59abe81d51e11bb8f83
1 file changed
+12
vllm/model_executor/model_loader/loader.py
+12
@@ -441,6 +441,18 @@ class DummyModelLoader(BaseModelLoader):
441
# NOTE(woosuk): For accurate performance evaluation, we assign
442
# random values to the weights.
443
initialize_dummy_weights(model)
444
+
445
+ for _, module in model.named_modules():
446
+ quant_method = getattr(module, "quant_method", None)
447
+ if quant_method is not None:
448
+ # When quant methods need to process weights after loading
449
+ # (for repacking, quantizing, etc), they expect parameters
450
+ # to be on the global target device. This scope is for the
451
+ # case where cpu offloading is used, where we will move the
452
+ # parameters onto device for processing and back off after.
453
+ with device_loading_context(
454
+ module, torch.device(device_config.device)):
455
+ quant_method.process_weights_after_loading(module)
456
return model.eval()
457
458