@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d2b2eed67c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Fix] Fix a condition for ignored sequences (#867)

Zhuohan Li committed Aug 27, 2023 at 23:00 UTC d2b2eed67c49cdda3c1d6fa09ee2ec128b318138
1 file changed +6 -6
vllm/core/scheduler.py
+6 -6
@@ -64,6 +64,9 @@ class Scheduler:
64 self.scheduler_config = scheduler_config
65 self.cache_config = cache_config
66
67 + self.prompt_limit = min(self.scheduler_config.max_model_len,
68 + self.scheduler_config.max_num_batched_tokens)
69 +
70 # Instantiate the scheduling policy.
71 self.policy = PolicyFactory.get_policy(policy_name="fcfs")
72 # Create the block space manager.
@@ -123,18 +126,15 @@ class Scheduler:
126 seq_group = self.waiting[0]
127
128 num_prompt_tokens = seq_group.get_seqs()[0].get_len()
126 - prompt_limit = min(
127 - self.scheduler_config.max_model_len,
128 - self.scheduler_config.max_num_batched_tokens)
129 - if num_prompt_tokens > prompt_limit:
129 + if num_prompt_tokens > self.prompt_limit:
130 logger.warning(
131 f"Input prompt ({num_prompt_tokens} tokens) is too long"
132 - f" and exceeds limit of {prompt_limit}")
132 + f" and exceeds limit of {self.prompt_limit}")
133 for seq in seq_group.get_seqs():
134 seq.status = SequenceStatus.FINISHED_IGNORED
135 ignored_seq_groups.append(seq_group)
136 self.waiting.pop(0)
137 - break
137 + continue
138
139 # If the sequence group cannot be allocated, stop.
140 if not self.block_manager.can_allocate(seq_group):