A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Misc] typo find in sampling_metadata.py (#10740)
wang.yuqi committed
Nov 29, 2024 at 13:17 UTC
c82b432d4a40fd6376a35fd38cb5fc37e9c53798
1 file changed
+1
vllm/model_executor/sampling_metadata.py
+1
@@ -454,6 +454,7 @@ class SamplingTensors:
454
if do_penalties:
455
for seq_group in sampling_metadata.seq_groups:
456
seq_ids = seq_group.seq_ids
457
+ sampling_params = seq_group.sampling_params
458
if (seq_group.is_prompt
459
and sampling_params.prompt_logprobs is not None):
460
prefill_len = len(seq_group.prompt_logprob_indices)