@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 94d2f59895
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Set replacement=True in torch.multinomial (#858)

Woosuk Kwon committed Aug 25, 2023 at 12:22 UTC 94d2f598956dbd5ab8a6097d6eaa8176035d3220
1 file changed +3 -1
vllm/model_executor/layers/sampler.py
+3 -1
@@ -302,7 +302,9 @@ def _sample_from_prompt(
302 # Random sampling.
303 # Sample `best_of` tokens for the prompt.
304 num_seqs = sampling_params.best_of
305 - next_token_ids = torch.multinomial(prob, num_samples=num_seqs)
305 + next_token_ids = torch.multinomial(prob,
306 + num_samples=num_seqs,
307 + replacement=True)
308 next_token_ids = next_token_ids.tolist()
309 return next_token_ids
310