A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Bugfix] Fix Mistral ChatCompletionRequest Body Exception (#16769)
Signed-off-by: Jasmond Loh <Jasmond.Loh@hotmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Jasmond L committed
Apr 25, 2025 at 22:26 UTC
d5615af9aee97ef44f46de722d48852eb5d40802
1 file changed
+32
-8
vllm/entrypoints/chat_utils.py
+32
-8
@@ -1198,14 +1198,25 @@ def apply_hf_chat_template(
1198
"allowed, so you must provide a chat template if the tokenizer "
1199
"does not define one.")
1200
1201
- return tokenizer.apply_chat_template(
1202
- conversation=conversation, # type: ignore[arg-type]
1203
- tools=tools, # type: ignore[arg-type]
1204
- chat_template=hf_chat_template,
1205
- tokenize=tokenize,
1206
- **kwargs,
1207
- )
1201
+ try:
1202
+
1203
+ return tokenizer.apply_chat_template(
1204
+ conversation=conversation, # type: ignore[arg-type]
1205
+ tools=tools, # type: ignore[arg-type]
1206
+ chat_template=hf_chat_template,
1207
+ tokenize=tokenize,
1208
+ **kwargs,
1209
+ )
1210
1211
+ # External library exceptions can sometimes occur despite the framework's
1212
+ # internal exception management capabilities.
1213
+ except Exception as e:
1214
+
1215
+ # Log and report any library-related exceptions for further
1216
+ # investigation.
1217
+ logger.exception(
1218
+ "An error occurred in `transformers` while applying chat template")
1219
+ raise ValueError from e
1220
1221
def apply_mistral_chat_template(
1222
tokenizer: MistralTokenizer,
@@ -1214,6 +1225,8 @@ def apply_mistral_chat_template(
1225
tools: Optional[list[dict[str, Any]]],
1226
**kwargs: Any,
1227
) -> list[int]:
1228
+ from mistral_common.exceptions import MistralCommonException
1229
+
1230
# The return value of resolve_mistral_chat_template is always None,
1231
# and we won't use it.
1232
resolve_mistral_chat_template(
@@ -1231,5 +1244,16 @@ def apply_mistral_chat_template(
1244
# if input does not comply with the expected format.
1245
# We convert those assertion errors to ValueErrors so they can be
1246
# are properly caught in the preprocessing_input step
1234
- except AssertionError as e:
1247
+ except (AssertionError, MistralCommonException) as e:
1248
+ raise ValueError from e
1249
+
1250
+ # External library exceptions can sometimes occur despite the framework's
1251
+ # internal exception management capabilities.
1252
+ except Exception as e:
1253
+
1254
+ # Log and report any library-related exceptions for further
1255
+ # investigation.
1256
+ logger.exception(
1257
+ "An error occurred in `mistral_common` while applying chat "
1258
+ "template")
1259
raise ValueError from e