A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Fix] Fix the condition of max_seq_len (#477)
Zhuohan Li committed
Jul 17, 2023 at 00:33 UTC
2bdea7ac110d3090d6a3c582aed36577ca480473
2 files changed
+2
-2
vllm/core/scheduler.py
+1
-1
@@ -190,7 +190,7 @@ class Scheduler:
190
break
191
192
num_prompt_tokens = seq_group.get_seqs()[0].get_len()
193
- if num_prompt_tokens >= self.scheduler_config.max_seq_len:
193
+ if num_prompt_tokens > self.scheduler_config.max_seq_len:
194
logger.warning(
195
f"Input prompt ({num_prompt_tokens} tokens) is too long"
196
" and exceeds limit of "
vllm/engine/llm_engine.py
+1
-1
@@ -300,7 +300,7 @@ class LLMEngine:
300
continue
301
302
# Check if the sequence has reached max_seq_len.
303
- if (seq.get_len() >=
303
+ if (seq.get_len() >
304
self.scheduler.scheduler_config.max_seq_len):
305
self.scheduler.free_seq(
306
seq, SequenceStatus.FINISHED_LENGTH_CAPPED)