A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Disable FlashInfer sampler by default (#26859)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Oct 14, 2025 at 22:35 UTC
e66d787bce22c56f995f4e2974e31ac020bc57ea
1 file changed
+6
-14
vllm/v1/sample/ops/topk_topp_sampler.py
+6
-14
@@ -46,23 +46,15 @@ class TopKTopPSampler(nn.Module):
46
"Falling back to default sampling implementation."
47
)
48
self.forward = self.forward_native
49
- elif envs.VLLM_USE_FLASHINFER_SAMPLER is not False:
50
- # NOTE(woosuk): The V0 sampler doesn't use FlashInfer for
51
- # sampling unless VLLM_USE_FLASHINFER_SAMPLER=1 (i.e., by
52
- # default it is unused). For backward compatibility, we set
53
- # `VLLM_USE_FLASHINFER_SAMPLER` as None by default and
54
- # interpret it differently in V0 and V1 samplers: In V0,
55
- # None means False, while in V1, None means True. This is
56
- # why we use the condition
57
- # `envs.VLLM_USE_FLASHINFER_SAMPLER is not False` here.
49
+ elif envs.VLLM_USE_FLASHINFER_SAMPLER:
50
+ # Users must opt in explicitly via VLLM_USE_FLASHINFER_SAMPLER=1.
51
logger.info_once("Using FlashInfer for top-p & top-k sampling.")
52
self.forward = self.forward_cuda
53
else:
61
- logger.warning_once(
62
- "FlashInfer is available, but it is not enabled. "
63
- "Falling back to the PyTorch-native implementation of "
64
- "top-p & top-k sampling. For the best performance, "
65
- "please set VLLM_USE_FLASHINFER_SAMPLER=1."
54
+ logger.debug_once(
55
+ "FlashInfer top-p/top-k sampling is available but disabled "
56
+ "by default. Set VLLM_USE_FLASHINFER_SAMPLER=1 to opt in "
57
+ "after verifying accuracy for your workloads."
58
)
59
self.forward = self.forward_native
60
else: