@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1e7d5c01f5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[misc] soft drop beam search (#8763)

youkaichao committed Sep 24, 2024 at 15:48 UTC 1e7d5c01f5c35424eede1bbe6f723dd8781120f0
2 files changed +10
vllm/envs.py
+5
@@ -62,6 +62,7 @@ if TYPE_CHECKING:
62 VLLM_TORCH_PROFILER_DIR: Optional[str] = None
63 VLLM_USE_TRITON_AWQ: bool = False
64 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
65 + VLLM_ALLOW_DEPRECATED_BEAM_SEARCH: bool = False
66
67
68 def get_default_cache_root():
@@ -195,6 +196,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
196 lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in
197 ("true", "1")),
198
199 + # If set, allowing the use of deprecated beam search implementation
200 + "VLLM_ALLOW_DEPRECATED_BEAM_SEARCH":
201 + lambda: os.environ.get("VLLM_ALLOW_DEPRECATED_BEAM_SEARCH", "0") == "1",
202 +
203 # Internal flag to enable Dynamo graph capture
204 "VLLM_TEST_DYNAMO_GRAPH_CAPTURE":
205 lambda: int(os.environ.get("VLLM_TEST_DYNAMO_GRAPH_CAPTURE", "0")),
vllm/sampling_params.py
+5
@@ -8,6 +8,7 @@ import msgspec
8 import torch
9 from typing_extensions import Annotated
10
11 +import vllm.envs as envs
12 from vllm.logger import init_logger
13
14 logger = init_logger(__name__)
@@ -260,6 +261,10 @@ class SamplingParams(
261
262 self._verify_args()
263 if self.use_beam_search:
264 + if not envs.VLLM_ALLOW_DEPRECATED_BEAM_SEARCH:
265 + raise ValueError(
266 + "Using beam search as a sampling parameter is deprecated, and will be removed in the future release. Please use the `vllm.LLM.use_beam_search` method for dedicated beam search instead, or set the environment variable `VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1` to suppress this error. For more details, see https://github.com/vllm-project/vllm/issues/8306 ." # noqa
267 + )
268 self._verify_beam_search()
269 else:
270 self._verify_non_beam_search()