A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Set replacement=True in torch.multinomial (#858)
Woosuk Kwon committed
Aug 25, 2023 at 12:22 UTC
94d2f598956dbd5ab8a6097d6eaa8176035d3220
1 file changed
+3
-1
vllm/model_executor/layers/sampler.py
+3
-1
@@ -302,7 +302,9 @@ def _sample_from_prompt(
302
# Random sampling.
303
# Sample `best_of` tokens for the prompt.
304
num_seqs = sampling_params.best_of
305
- next_token_ids = torch.multinomial(prob, num_samples=num_seqs)
305
+ next_token_ids = torch.multinomial(prob,
306
+ num_samples=num_seqs,
307
+ replacement=True)
308
next_token_ids = next_token_ids.tolist()
309
return next_token_ids
310