A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
fix max seq len (#489)
Lily Liu committed
Jul 17, 2023 at 23:20 UTC
b4b195b3608610533e9d2c6c0168be71a8436355
4 files changed
+8
-8
vllm/config.py
+2
-2
@@ -204,10 +204,10 @@ class SchedulerConfig:
204
"""
205
206
def __init__(self, max_num_batched_tokens: int, max_num_seqs: int,
207
- max_seq_len: int) -> None:
207
+ max_model_len: int) -> None:
208
self.max_num_batched_tokens = max_num_batched_tokens
209
self.max_num_seqs = max_num_seqs
210
- self.max_seq_len = max_seq_len
210
+ self.max_model_len = max_model_len
211
212
213
_STR_DTYPE_TO_TORCH_DTYPE = {
vllm/core/scheduler.py
+3
-1
@@ -190,7 +190,9 @@ class Scheduler:
190
break
191
192
num_prompt_tokens = seq_group.get_seqs()[0].get_len()
193
- if num_prompt_tokens > self.scheduler_config.max_seq_len:
193
+ if num_prompt_tokens > min(
194
+ self.scheduler_config.max_model_len,
195
+ self.scheduler_config.max_num_batched_tokens):
196
logger.warning(
197
f"Input prompt ({num_prompt_tokens} tokens) is too long"
198
" and exceeds limit of "
vllm/engine/arg_utils.py
+2
-3
@@ -155,11 +155,10 @@ class EngineArgs:
155
parallel_config = ParallelConfig(self.pipeline_parallel_size,
156
self.tensor_parallel_size,
157
self.worker_use_ray)
158
- model_max_len = getattr(model_config.hf_config,
158
+ max_model_len = getattr(model_config.hf_config,
159
'max_position_embeddings', float('inf'))
160
- max_seq_len = min(self.max_num_batched_tokens, model_max_len)
160
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
162
- self.max_num_seqs, max_seq_len)
161
+ self.max_num_seqs, max_model_len)
162
return model_config, cache_config, parallel_config, scheduler_config
163
164
vllm/engine/llm_engine.py
+1
-2
@@ -300,8 +300,7 @@ class LLMEngine:
300
continue
301
302
# Check if the sequence has reached max_seq_len.
303
- if (seq.get_len() >
304
- self.scheduler.scheduler_config.max_seq_len):
303
+ if seq.get_len() > self.scheduler_config.max_model_len:
304
self.scheduler.free_seq(
305
seq, SequenceStatus.FINISHED_LENGTH_CAPPED)
306
continue