@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6d70198b17
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Doc] Fix typo (#11666)

Signed-off-by: Kazuhiro Serizawa <nserihiro@gmail.com>

Kazuhiro Serizawa committed Jan 1, 2025 at 17:10 UTC 6d70198b17b008f5b845582590b96a507b4d68b5
2 files changed +2 -2
vllm/model_executor/layers/rejection_sampler.py
+1 -1
@@ -39,7 +39,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
39 strict_mode: Whether or not to perform shape/device/dtype checks
40 during sampling. This catches correctness issues but adds
41 nontrivial latency.
42 - use_falshinfer: We will use this parameter to determine whether
42 + use_flashinfer: We will use this parameter to determine whether
43 to use the FlashInfer rejection sampling kernel or not. If it's
44 None, we will use the default value from the environment variable.
45 This parameter is only used for testing purposes.
vllm/v1/sample/ops/topk_topp_sampler.py
+1 -1
@@ -44,7 +44,7 @@ class TopKTopPSampler(nn.Module):
44 logger.warning(
45 "FlashInfer is not available. Falling back to the PyTorch-"
46 "native implementation of top-p & top-k sampling. For the "
47 - "best performance, please install FalshInfer.")
47 + "best performance, please install FlashInfer.")
48 self.forward = self.forward_native
49 else:
50 self.forward = self.forward_native