@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 090e945e36
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Frontend] Make beam search emulator temperature modifiable (#8928)

Co-authored-by: Eduard Balzin <nfunctor@yahoo.fr>

Edouard B. committed Sep 28, 2024 at 20:30 UTC 090e945e36cfe849b484db5414f64df96e97d678
1 file changed +3 -1
vllm/entrypoints/llm.py
+3 -1
@@ -396,6 +396,7 @@ class LLM:
396 beam_width: int,
397 max_tokens: int,
398 ignore_eos: bool = False,
399 + temperature: float = 0.0,
400 ) -> List[BeamSearchOutput]:
401 """
402 Generate sequences using beam search.
@@ -405,6 +406,7 @@ class LLM:
406 of token IDs.
407 beam_width: The number of beams to keep at each step.
408 max_tokens: The max number of tokens to generate for each prompt.
409 + temperature: The temperature to use for generation.
410
411 TODO: how does beam search work together with length penalty, frequency
412 penalty, and stopping criteria, etc.?
@@ -416,7 +418,7 @@ class LLM:
418 # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
419 beam_search_params = SamplingParams(logprobs=2 * beam_width,
420 max_tokens=1,
419 - temperature=0.0)
421 + temperature=temperature)
422 instances: List[BeamSearchInstance] = []
423
424 for prompt in prompts: