A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[BugFix]: properly catch templating error when preprocess input (#13976)
Signed-off-by: Guillaume Calmettes <gcalmettes@scaleway.com>
Guillaume Calmettes committed
Mar 14, 2025 at 08:58 UTC
fd8e055ffba508e094cd1793e49bbdc5e53b7266
5 files changed
+37
vllm/entrypoints/openai/serving_chat.py
+10
@@ -7,6 +7,7 @@ from collections.abc import AsyncGenerator, AsyncIterator
7
from collections.abc import Sequence as GenericSequence
8
from typing import Callable, Final, Optional, Union
9
10
+import jinja2
11
from fastapi import Request
12
13
from vllm.config import ModelConfig
@@ -199,6 +200,15 @@ class OpenAIServingChat(OpenAIServing):
200
except ValueError as e:
201
logger.exception("Error in preprocessing prompt inputs")
202
return self.create_error_response(str(e))
203
+ except TypeError as e:
204
+ logger.exception("Error in preprocessing prompt inputs")
205
+ return self.create_error_response(str(e))
206
+ except RuntimeError as e:
207
+ logger.exception("Error in preprocessing prompt inputs")
208
+ return self.create_error_response(str(e))
209
+ except jinja2.TemplateError as e:
210
+ logger.exception("Error in preprocessing prompt inputs")
211
+ return self.create_error_response(str(e))
212
213
request_id = "chatcmpl-" \
214
f"{self._base_request_id(raw_request, request.request_id)}"
vllm/entrypoints/openai/serving_completion.py
+10
@@ -6,6 +6,7 @@ from collections.abc import AsyncGenerator, AsyncIterator
6
from collections.abc import Sequence as GenericSequence
7
from typing import Optional, Union, cast
8
9
+import jinja2
10
from fastapi import Request
11
12
from vllm.config import ModelConfig
@@ -114,6 +115,15 @@ class OpenAIServingCompletion(OpenAIServing):
115
except ValueError as e:
116
logger.exception("Error in preprocessing prompt inputs")
117
return self.create_error_response(str(e))
118
+ except TypeError as e:
119
+ logger.exception("Error in preprocessing prompt inputs")
120
+ return self.create_error_response(str(e))
121
+ except RuntimeError as e:
122
+ logger.exception("Error in preprocessing prompt inputs")
123
+ return self.create_error_response(str(e))
124
+ except jinja2.TemplateError as e:
125
+ logger.exception("Error in preprocessing prompt inputs")
126
+ return self.create_error_response(str(e))
127
128
# Schedule the request and get the result generator.
129
generators: list[AsyncGenerator[RequestOutput, None]] = []
vllm/entrypoints/openai/serving_embedding.py
+3
@@ -142,6 +142,9 @@ class OpenAIServingEmbedding(OpenAIServing):
142
except ValueError as e:
143
logger.exception("Error in preprocessing prompt inputs")
144
return self.create_error_response(str(e))
145
+ except TypeError as e:
146
+ logger.exception("Error in preprocessing prompt inputs")
147
+ return self.create_error_response(str(e))
148
149
# Schedule the request and get the result generator.
150
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
vllm/entrypoints/openai/serving_pooling.py
+7
@@ -6,6 +6,7 @@ import time
6
from collections.abc import AsyncGenerator
7
from typing import Final, Literal, Optional, Union, cast
8
9
+import jinja2
10
import numpy as np
11
from fastapi import Request
12
from typing_extensions import assert_never
@@ -138,6 +139,12 @@ class OpenAIServingPooling(OpenAIServing):
139
except ValueError as e:
140
logger.exception("Error in preprocessing prompt inputs")
141
return self.create_error_response(str(e))
142
+ except TypeError as e:
143
+ logger.exception("Error in preprocessing prompt inputs")
144
+ return self.create_error_response(str(e))
145
+ except jinja2.TemplateError as e:
146
+ logger.exception("Error in preprocessing prompt inputs")
147
+ return self.create_error_response(str(e))
148
149
# Schedule the request and get the result generator.
150
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
vllm/entrypoints/openai/serving_tokenization.py
+7
@@ -2,6 +2,7 @@
2
3
from typing import Final, Optional, Union
4
5
+import jinja2
6
from fastapi import Request
7
8
from vllm.config import ModelConfig
@@ -91,6 +92,12 @@ class OpenAIServingTokenization(OpenAIServing):
92
except ValueError as e:
93
logger.exception("Error in preprocessing prompt inputs")
94
return self.create_error_response(str(e))
95
+ except TypeError as e:
96
+ logger.exception("Error in preprocessing prompt inputs")
97
+ return self.create_error_response(str(e))
98
+ except jinja2.TemplateError as e:
99
+ logger.exception("Error in preprocessing prompt inputs")
100
+ return self.create_error_response(str(e))
101
102
input_ids: list[int] = []
103
for i, engine_prompt in enumerate(engine_prompts):