@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 66c54aa9c3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Check the max prompt length for the OpenAI completions API (#472)

Nicolas Basile committed Aug 8, 2023 at 17:43 UTC 66c54aa9c33555a6b41421d57d3ad6c1bf004ec9
1 file changed +12 -5
vllm/entrypoints/openai/api_server.py
+12 -5
@@ -120,7 +120,7 @@ async def check_length(request, prompt):
120 token_num = len(input_ids)
121
122 if token_num + request.max_tokens > max_model_len:
123 - return create_error_response(
123 + return input_ids, create_error_response(
124 HTTPStatus.BAD_REQUEST,
125 f"This model's maximum context length is {max_model_len} tokens. "
126 f"However, you requested {request.max_tokens + token_num} tokens "
@@ -129,7 +129,7 @@ async def check_length(request, prompt):
129 f"Please reduce the length of the messages or completion.",
130 )
131 else:
132 - return None
132 + return input_ids, None
133
134
135 @app.get("/v1/models")
@@ -191,7 +191,7 @@ async def create_chat_completion(raw_request: Request):
191 "logit_bias is not currently supported")
192
193 prompt = await get_gen_prompt(request)
194 - error_check_ret = await check_length(request, prompt)
194 + token_ids, error_check_ret = await check_length(request, prompt)
195 if error_check_ret is not None:
196 return error_check_ret
197
@@ -215,7 +215,8 @@ async def create_chat_completion(raw_request: Request):
215 except ValueError as e:
216 return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
217
218 - result_generator = engine.generate(prompt, sampling_params, request_id)
218 + result_generator = engine.generate(prompt, sampling_params, request_id,
219 + token_ids)
220
221 async def abort_request() -> None:
222 await engine.abort(request_id)
@@ -386,6 +387,11 @@ async def create_completion(raw_request: Request):
387 prompt = request.prompt[0]
388 else:
389 prompt = request.prompt
390 +
391 + token_ids, error_check_ret = await check_length(request, prompt)
392 + if error_check_ret is not None:
393 + return error_check_ret
394 +
395 created_time = int(time.time())
396 try:
397 sampling_params = SamplingParams(
@@ -405,7 +411,8 @@ async def create_completion(raw_request: Request):
411 except ValueError as e:
412 return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
413
408 - result_generator = engine.generate(prompt, sampling_params, request_id)
414 + result_generator = engine.generate(prompt, sampling_params, request_id,
415 + token_ids)
416
417 # Similar to the OpenAI API, when n != best_of, we do not stream the
418 # results. In addition, we do not stream the results when use beam search.