A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Doc] Fix typo (#11666)
Signed-off-by: Kazuhiro Serizawa <nserihiro@gmail.com>
Kazuhiro Serizawa committed
Jan 1, 2025 at 17:10 UTC
6d70198b17b008f5b845582590b96a507b4d68b5
2 files changed
+2
-2
vllm/model_executor/layers/rejection_sampler.py
+1
-1
@@ -39,7 +39,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
39
strict_mode: Whether or not to perform shape/device/dtype checks
40
during sampling. This catches correctness issues but adds
41
nontrivial latency.
42
- use_falshinfer: We will use this parameter to determine whether
42
+ use_flashinfer: We will use this parameter to determine whether
43
to use the FlashInfer rejection sampling kernel or not. If it's
44
None, we will use the default value from the environment variable.
45
This parameter is only used for testing purposes.
vllm/v1/sample/ops/topk_topp_sampler.py
+1
-1
@@ -44,7 +44,7 @@ class TopKTopPSampler(nn.Module):
44
logger.warning(
45
"FlashInfer is not available. Falling back to the PyTorch-"
46
"native implementation of top-p & top-k sampling. For the "
47
- "best performance, please install FalshInfer.")
47
+ "best performance, please install FlashInfer.")
48
self.forward = self.forward_native
49
else:
50
self.forward = self.forward_native