@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c82b432d4a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Misc] typo find in sampling_metadata.py (#10740)

wang.yuqi committed Nov 29, 2024 at 13:17 UTC c82b432d4a40fd6376a35fd38cb5fc37e9c53798
1 file changed +1
vllm/model_executor/sampling_metadata.py
+1
@@ -454,6 +454,7 @@ class SamplingTensors:
454 if do_penalties:
455 for seq_group in sampling_metadata.seq_groups:
456 seq_ids = seq_group.seq_ids
457 + sampling_params = seq_group.sampling_params
458 if (seq_group.is_prompt
459 and sampling_params.prompt_logprobs is not None):
460 prefill_len = len(seq_group.prompt_logprob_indices)