@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 82f3937e59
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Misc] add process_weights_after_loading for DummyLoader (#8969)

Divakar Verma committed Sep 30, 2024 at 22:46 UTC 82f3937e599a4f088a62e59abe81d51e11bb8f83
1 file changed +12
vllm/model_executor/model_loader/loader.py
+12
@@ -441,6 +441,18 @@ class DummyModelLoader(BaseModelLoader):
441 # NOTE(woosuk): For accurate performance evaluation, we assign
442 # random values to the weights.
443 initialize_dummy_weights(model)
444 +
445 + for _, module in model.named_modules():
446 + quant_method = getattr(module, "quant_method", None)
447 + if quant_method is not None:
448 + # When quant methods need to process weights after loading
449 + # (for repacking, quantizing, etc), they expect parameters
450 + # to be on the global target device. This scope is for the
451 + # case where cpu offloading is used, where we will move the
452 + # parameters onto device for processing and back off after.
453 + with device_loading_context(
454 + module, torch.device(device_config.device)):
455 + quant_method.process_weights_after_loading(module)
456 return model.eval()
457
458