@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b4b195b360
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

fix max seq len (#489)

Lily Liu committed Jul 17, 2023 at 23:20 UTC b4b195b3608610533e9d2c6c0168be71a8436355
4 files changed +8 -8
vllm/config.py
+2 -2
@@ -204,10 +204,10 @@ class SchedulerConfig:
204 """
205
206 def __init__(self, max_num_batched_tokens: int, max_num_seqs: int,
207 - max_seq_len: int) -> None:
207 + max_model_len: int) -> None:
208 self.max_num_batched_tokens = max_num_batched_tokens
209 self.max_num_seqs = max_num_seqs
210 - self.max_seq_len = max_seq_len
210 + self.max_model_len = max_model_len
211
212
213 _STR_DTYPE_TO_TORCH_DTYPE = {
vllm/core/scheduler.py
+3 -1
@@ -190,7 +190,9 @@ class Scheduler:
190 break
191
192 num_prompt_tokens = seq_group.get_seqs()[0].get_len()
193 - if num_prompt_tokens > self.scheduler_config.max_seq_len:
193 + if num_prompt_tokens > min(
194 + self.scheduler_config.max_model_len,
195 + self.scheduler_config.max_num_batched_tokens):
196 logger.warning(
197 f"Input prompt ({num_prompt_tokens} tokens) is too long"
198 " and exceeds limit of "
vllm/engine/arg_utils.py
+2 -3
@@ -155,11 +155,10 @@ class EngineArgs:
155 parallel_config = ParallelConfig(self.pipeline_parallel_size,
156 self.tensor_parallel_size,
157 self.worker_use_ray)
158 - model_max_len = getattr(model_config.hf_config,
158 + max_model_len = getattr(model_config.hf_config,
159 'max_position_embeddings', float('inf'))
160 - max_seq_len = min(self.max_num_batched_tokens, model_max_len)
160 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
162 - self.max_num_seqs, max_seq_len)
161 + self.max_num_seqs, max_model_len)
162 return model_config, cache_config, parallel_config, scheduler_config
163
164
vllm/engine/llm_engine.py
+1 -2
@@ -300,8 +300,7 @@ class LLMEngine:
300 continue
301
302 # Check if the sequence has reached max_seq_len.
303 - if (seq.get_len() >
304 - self.scheduler.scheduler_config.max_seq_len):
303 + if seq.get_len() > self.scheduler_config.max_model_len:
304 self.scheduler.free_seq(
305 seq, SequenceStatus.FINISHED_LENGTH_CAPPED)
306 continue