A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
fix doc (#622)
Chaofan Lin committed
Aug 1, 2023 at 04:11 UTC
aa39e42c5a8a2359363529571cb553cc30e26d58
2 files changed
+2
-2
vllm/config.py
+1
-1
@@ -224,7 +224,7 @@ class SchedulerConfig:
224
a single iteration.
225
max_num_seqs: Maximum number of sequences to be processed in a single
226
iteration.
227
- max_seq_len: Maximum length of a sequence (including prompt
227
+ max_model_len: Maximum length of a sequence (including prompt
228
and generated text).
229
"""
230
vllm/engine/llm_engine.py
+1
-1
@@ -353,7 +353,7 @@ class LLMEngine:
353
if stopped:
354
continue
355
356
- # Check if the sequence has reached max_seq_len.
356
+ # Check if the sequence has reached max_model_len.
357
if seq.get_len() > self.scheduler_config.max_model_len:
358
self.scheduler.free_seq(
359
seq, SequenceStatus.FINISHED_LENGTH_CAPPED)