@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aa39e42c5a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

fix doc (#622)

Chaofan Lin committed Aug 1, 2023 at 04:11 UTC aa39e42c5a8a2359363529571cb553cc30e26d58
2 files changed +2 -2
vllm/config.py
+1 -1
@@ -224,7 +224,7 @@ class SchedulerConfig:
224 a single iteration.
225 max_num_seqs: Maximum number of sequences to be processed in a single
226 iteration.
227 - max_seq_len: Maximum length of a sequence (including prompt
227 + max_model_len: Maximum length of a sequence (including prompt
228 and generated text).
229 """
230
vllm/engine/llm_engine.py
+1 -1
@@ -353,7 +353,7 @@ class LLMEngine:
353 if stopped:
354 continue
355
356 - # Check if the sequence has reached max_seq_len.
356 + # Check if the sequence has reached max_model_len.
357 if seq.get_len() > self.scheduler_config.max_model_len:
358 self.scheduler.free_seq(
359 seq, SequenceStatus.FINISHED_LENGTH_CAPPED)