A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 21:00 UTC
[CI]revert initialize_model context manager (#38426)
Signed-off-by: Kunshang Ji <jikunshang95@gmail.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
Kunshang Ji committed
Mar 29, 2026 at 00:56 UTC
aa4eb0db78ec469438a7a18147b0372fe2eb7cf4
1 file changed
+8
-4
vllm/model_executor/model_loader/base_loader.py
+8
-4
@@ -50,10 +50,14 @@ class BaseModelLoader(ABC):
50
device_config.device if load_config.device is None else load_config.device
51
)
52
target_device = torch.device(load_device)
53
- with set_default_torch_dtype(model_config.dtype), target_device:
54
- model = initialize_model(
55
- vllm_config=vllm_config, model_config=model_config, prefix=prefix
56
- )
53
+ with set_default_torch_dtype(model_config.dtype):
54
+ with target_device:
55
+ model = initialize_model(
56
+ vllm_config=vllm_config,
57
+ model_config=model_config,
58
+ prefix=prefix,
59
+ )
60
+
61
log_model_inspection(model)
62
63
logger.debug("Loading weights on %s ...", load_device)