A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Fix] Fix a condition for ignored sequences (#867)
Zhuohan Li committed
Aug 27, 2023 at 23:00 UTC
d2b2eed67c49cdda3c1d6fa09ee2ec128b318138
1 file changed
+6
-6
vllm/core/scheduler.py
+6
-6
@@ -64,6 +64,9 @@ class Scheduler:
64
self.scheduler_config = scheduler_config
65
self.cache_config = cache_config
66
67
+ self.prompt_limit = min(self.scheduler_config.max_model_len,
68
+ self.scheduler_config.max_num_batched_tokens)
69
+
70
# Instantiate the scheduling policy.
71
self.policy = PolicyFactory.get_policy(policy_name="fcfs")
72
# Create the block space manager.
@@ -123,18 +126,15 @@ class Scheduler:
126
seq_group = self.waiting[0]
127
128
num_prompt_tokens = seq_group.get_seqs()[0].get_len()
126
- prompt_limit = min(
127
- self.scheduler_config.max_model_len,
128
- self.scheduler_config.max_num_batched_tokens)
129
- if num_prompt_tokens > prompt_limit:
129
+ if num_prompt_tokens > self.prompt_limit:
130
logger.warning(
131
f"Input prompt ({num_prompt_tokens} tokens) is too long"
132
- f" and exceeds limit of {prompt_limit}")
132
+ f" and exceeds limit of {self.prompt_limit}")
133
for seq in seq_group.get_seqs():
134
seq.status = SequenceStatus.FINISHED_IGNORED
135
ignored_seq_groups.append(seq_group)
136
self.waiting.pop(0)
137
- break
137
+ continue
138
139
# If the sequence group cannot be allocated, stop.
140
if not self.block_manager.can_allocate(seq_group):