A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 16:15 UTC
[Chore] Fix torch precision warning (#30428)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Dec 10, 2025 at 23:05 UTC
d6464f267979946a1c2d9c6029ef2007be73ca09
3 files changed
+9
-7
tests/v1/e2e/test_async_scheduling.py
+2
-2
@@ -152,8 +152,8 @@ def run_tests(
152
m.setenv("VLLM_ATTENTION_BACKEND", "ROCM_AITER_FA")
153
else:
154
m.setenv("VLLM_ATTENTION_BACKEND", "FLEX_ATTENTION")
155
- # lock matmul precision to full FP32
156
- m.setenv("VLLM_FLOAT32_MATMUL_PRECISION", "highest")
155
+ # lock matmul precision to full FP32 (IEEE)
156
+ m.setenv("VLLM_FLOAT32_MATMUL_PRECISION", "ieee")
157
# m.setenv("VLLM_BATCH_INVARIANT", "1")
158
outputs: list[tuple[str, list, list]] = []
159
for n, (
vllm/envs.py
+6
-4
@@ -74,7 +74,7 @@ if TYPE_CHECKING:
74
VLLM_MEDIA_CONNECTOR: str = "http"
75
VLLM_TARGET_DEVICE: str = "cuda"
76
VLLM_MAIN_CUDA_VERSION: str = "12.9"
77
- VLLM_FLOAT32_MATMUL_PRECISION: Literal["highest", "high", "medium"] = "highest"
77
+ VLLM_FLOAT32_MATMUL_PRECISION: Literal["ieee", "tf32"] = "ieee"
78
MAX_JOBS: str | None = None
79
NVCC_THREADS: str | None = None
80
VLLM_USE_PRECOMPILED: bool = False
@@ -456,11 +456,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
456
"VLLM_MAIN_CUDA_VERSION": lambda: os.getenv("VLLM_MAIN_CUDA_VERSION", "").lower()
457
or "12.9",
458
# Controls PyTorch float32 matmul precision mode within vLLM workers.
459
- # Valid options mirror torch.set_float32_matmul_precision
459
+ # Accepted values:
460
+ # - "ieee" (default): force full IEEE FP32 matmul precision.
461
+ # - "tf32": enable TensorFloat32-based fast matmul.
462
"VLLM_FLOAT32_MATMUL_PRECISION": env_with_choices(
463
"VLLM_FLOAT32_MATMUL_PRECISION",
462
- "highest",
463
- ["highest", "high", "medium"],
464
+ "ieee",
465
+ ["ieee", "tf32"],
466
case_sensitive=False,
467
),
468
# Maximum number of compilation jobs to run in parallel.
vllm/v1/worker/gpu_worker.py
+1
-1
@@ -81,7 +81,7 @@ class Worker(WorkerBase):
81
82
# configure float32 matmul precision according to vLLM env.
83
precision = envs.VLLM_FLOAT32_MATMUL_PRECISION
84
- torch.set_float32_matmul_precision(precision)
84
+ torch.backends.cuda.matmul.fp32_precision = precision
85
86
if self.model_config.trust_remote_code:
87
# note: lazy import to avoid importing torch before initializing