A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Bugfix] Fix boolean conversion for OpenVINO env variable (#13615)
Helena Kloosterman committed
Feb 22, 2025 at 17:04 UTC
382f66fb0857f6febd8ff36ff9b8a28b1a840f2b
2 files changed
+5
-3
vllm/envs.py
+3
-2
@@ -360,8 +360,9 @@ environment_variables: Dict[str, Callable[[], Any]] = {
360
# Enables weights compression during model export via HF Optimum
361
# default is False
362
"VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS":
363
- lambda: bool(os.getenv("VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS", False)),
364
-
363
+ lambda:
364
+ (os.environ.get("VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS", "0").lower() in
365
+ ("on", "true", "1")),
366
# If the env var is set, then all workers will execute as separate
367
# processes from the engine, and we use the same mechanism to trigger
368
# execution on all workers.
vllm/model_executor/model_loader/openvino.py
+2
-1
@@ -125,7 +125,8 @@ class OpenVINOCausalLM(nn.Module):
125
"as-is, all possible options that may affect model conversion "
126
"are ignored.")
127
128
- load_in_8bit = envs.VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS
128
+ load_in_8bit = (envs.VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS
129
+ if export else False)
130
pt_model = OVModelForCausalLM.from_pretrained(
131
model_config.model,
132
export=export,