@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d3c8180ac4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Fixing max token error message for openai compatible server (#4016)

Jack Gordley committed Apr 23, 2024 at 12:06 UTC d3c8180ac4143f4affd2ef26855058e96b72b5f5
1 file changed +6
vllm/entrypoints/openai/serving_engine.py
+6
@@ -206,6 +206,12 @@ class OpenAIServing:
206 token_num = len(input_ids)
207
208 if request.max_tokens is None:
209 + if token_num >= self.max_model_len:
210 + raise ValueError(
211 + f"This model's maximum context length is "
212 + f"{self.max_model_len} tokens. However, you requested "
213 + f"{token_num} tokens in the messages, "
214 + f"Please reduce the length of the messages.", )
215 request.max_tokens = self.max_model_len - token_num
216
217 if token_num + request.max_tokens > self.max_model_len: