A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 17:45 UTC
[Bug] Revert torch warning fix (#31585)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Jan 5, 2026 at 17:31 UTC
af9a7ec2558293981fc984d064dfca4887c8033c
3 files changed
+6
-8
tests/v1/e2e/test_async_scheduling.py
+1
-1
@@ -154,7 +154,7 @@ def run_tests(
154
155
with monkeypatch.context() as m:
156
# lock matmul precision to full FP32 (IEEE)
157
- m.setenv("VLLM_FLOAT32_MATMUL_PRECISION", "ieee")
157
+ m.setenv("VLLM_FLOAT32_MATMUL_PRECISION", "highest")
158
# m.setenv("VLLM_BATCH_INVARIANT", "1")
159
outputs: list[tuple[str, list, list]] = []
160
for n, (
vllm/envs.py
+4
-6
@@ -75,7 +75,7 @@ if TYPE_CHECKING:
75
VLLM_MEDIA_CONNECTOR: str = "http"
76
VLLM_TARGET_DEVICE: str = "cuda"
77
VLLM_MAIN_CUDA_VERSION: str = "12.9"
78
- VLLM_FLOAT32_MATMUL_PRECISION: Literal["ieee", "tf32"] = "ieee"
78
+ VLLM_FLOAT32_MATMUL_PRECISION: Literal["highest", "high", "medium"] = "highest"
79
MAX_JOBS: str | None = None
80
NVCC_THREADS: str | None = None
81
VLLM_USE_PRECOMPILED: bool = False
@@ -459,13 +459,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
459
"VLLM_MAIN_CUDA_VERSION": lambda: os.getenv("VLLM_MAIN_CUDA_VERSION", "").lower()
460
or "12.9",
461
# Controls PyTorch float32 matmul precision mode within vLLM workers.
462
- # Accepted values:
463
- # - "ieee" (default): force full IEEE FP32 matmul precision.
464
- # - "tf32": enable TensorFloat32-based fast matmul.
462
+ # Valid options mirror torch.set_float32_matmul_precision
463
"VLLM_FLOAT32_MATMUL_PRECISION": env_with_choices(
464
"VLLM_FLOAT32_MATMUL_PRECISION",
467
- "ieee",
468
- ["ieee", "tf32"],
465
+ "highest",
466
+ ["highest", "high", "medium"],
467
case_sensitive=False,
468
),
469
# Maximum number of compilation jobs to run in parallel.
vllm/v1/worker/gpu_worker.py
+1
-1
@@ -84,7 +84,7 @@ class Worker(WorkerBase):
84
85
# configure float32 matmul precision according to vLLM env.
86
precision = envs.VLLM_FLOAT32_MATMUL_PRECISION
87
- torch.backends.cuda.matmul.fp32_precision = precision
87
+ torch.set_float32_matmul_precision(precision)
88
89
if self.model_config.trust_remote_code:
90
# note: lazy import to avoid importing torch before initializing