A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Check the max prompt length for the OpenAI completions API (#472)
Nicolas Basile committed
Aug 8, 2023 at 17:43 UTC
66c54aa9c33555a6b41421d57d3ad6c1bf004ec9
1 file changed
+12
-5
vllm/entrypoints/openai/api_server.py
+12
-5
@@ -120,7 +120,7 @@ async def check_length(request, prompt):
120
token_num = len(input_ids)
121
122
if token_num + request.max_tokens > max_model_len:
123
- return create_error_response(
123
+ return input_ids, create_error_response(
124
HTTPStatus.BAD_REQUEST,
125
f"This model's maximum context length is {max_model_len} tokens. "
126
f"However, you requested {request.max_tokens + token_num} tokens "
@@ -129,7 +129,7 @@ async def check_length(request, prompt):
129
f"Please reduce the length of the messages or completion.",
130
)
131
else:
132
- return None
132
+ return input_ids, None
133
134
135
@app.get("/v1/models")
@@ -191,7 +191,7 @@ async def create_chat_completion(raw_request: Request):
191
"logit_bias is not currently supported")
192
193
prompt = await get_gen_prompt(request)
194
- error_check_ret = await check_length(request, prompt)
194
+ token_ids, error_check_ret = await check_length(request, prompt)
195
if error_check_ret is not None:
196
return error_check_ret
197
@@ -215,7 +215,8 @@ async def create_chat_completion(raw_request: Request):
215
except ValueError as e:
216
return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
217
218
- result_generator = engine.generate(prompt, sampling_params, request_id)
218
+ result_generator = engine.generate(prompt, sampling_params, request_id,
219
+ token_ids)
220
221
async def abort_request() -> None:
222
await engine.abort(request_id)
@@ -386,6 +387,11 @@ async def create_completion(raw_request: Request):
387
prompt = request.prompt[0]
388
else:
389
prompt = request.prompt
390
+
391
+ token_ids, error_check_ret = await check_length(request, prompt)
392
+ if error_check_ret is not None:
393
+ return error_check_ret
394
+
395
created_time = int(time.time())
396
try:
397
sampling_params = SamplingParams(
@@ -405,7 +411,8 @@ async def create_completion(raw_request: Request):
411
except ValueError as e:
412
return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
413
408
- result_generator = engine.generate(prompt, sampling_params, request_id)
414
+ result_generator = engine.generate(prompt, sampling_params, request_id,
415
+ token_ids)
416
417
# Similar to the OpenAI API, when n != best_of, we do not stream the
418
# results. In addition, we do not stream the results when use beam search.