A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[BugFix] Prevent `LLM.encode` for non-generation Models (#5184)
Co-authored-by: mgoin <michael@neuralmagic.com>
Robert Shaw committed
Jun 1, 2024 at 19:35 UTC
044793d8df6aeb5326b5992d0e60aa4457760e8a
1 file changed
+10
vllm/entrypoints/llm.py
+10
@@ -276,6 +276,11 @@ class LLM:
276
considered legacy and may be deprecated in the future. You should
277
instead pass them via the ``inputs`` parameter.
278
"""
279
+ if self.llm_engine.model_config.embedding_mode:
280
+ raise ValueError(
281
+ "LLM.generate() is only supported for generation models "
282
+ "(XForCausalLM).")
283
+
284
if prompt_token_ids is not None or multi_modal_data is not None:
285
inputs = self._convert_v1_inputs(
286
prompts=cast(Optional[Union[str, List[str]]], prompts),
@@ -420,6 +425,11 @@ class LLM:
425
considered legacy and may be deprecated in the future. You should
426
instead pass them via the ``inputs`` parameter.
427
"""
428
+ if not self.llm_engine.model_config.embedding_mode:
429
+ raise ValueError(
430
+ "LLM.encode() is only supported for embedding models (XModel)."
431
+ )
432
+
433
if prompt_token_ids is not None or multi_modal_data is not None:
434
inputs = self._convert_v1_inputs(
435
prompts=cast(Optional[Union[str, List[str]]], prompts),