@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2bdea7ac11
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Fix] Fix the condition of max_seq_len (#477)

Zhuohan Li committed Jul 17, 2023 at 00:33 UTC 2bdea7ac110d3090d6a3c582aed36577ca480473
2 files changed +2 -2
vllm/core/scheduler.py
+1 -1
@@ -190,7 +190,7 @@ class Scheduler:
190 break
191
192 num_prompt_tokens = seq_group.get_seqs()[0].get_len()
193 - if num_prompt_tokens >= self.scheduler_config.max_seq_len:
193 + if num_prompt_tokens > self.scheduler_config.max_seq_len:
194 logger.warning(
195 f"Input prompt ({num_prompt_tokens} tokens) is too long"
196 " and exceeds limit of "
vllm/engine/llm_engine.py
+1 -1
@@ -300,7 +300,7 @@ class LLMEngine:
300 continue
301
302 # Check if the sequence has reached max_seq_len.
303 - if (seq.get_len() >=
303 + if (seq.get_len() >
304 self.scheduler.scheduler_config.max_seq_len):
305 self.scheduler.free_seq(
306 seq, SequenceStatus.FINISHED_LENGTH_CAPPED)