@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 044793d8df
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[BugFix] Prevent `LLM.encode` for non-generation Models (#5184)

Co-authored-by: mgoin <michael@neuralmagic.com>

Robert Shaw committed Jun 1, 2024 at 19:35 UTC 044793d8df6aeb5326b5992d0e60aa4457760e8a
1 file changed +10
vllm/entrypoints/llm.py
+10
@@ -276,6 +276,11 @@ class LLM:
276 considered legacy and may be deprecated in the future. You should
277 instead pass them via the ``inputs`` parameter.
278 """
279 + if self.llm_engine.model_config.embedding_mode:
280 + raise ValueError(
281 + "LLM.generate() is only supported for generation models "
282 + "(XForCausalLM).")
283 +
284 if prompt_token_ids is not None or multi_modal_data is not None:
285 inputs = self._convert_v1_inputs(
286 prompts=cast(Optional[Union[str, List[str]]], prompts),
@@ -420,6 +425,11 @@ class LLM:
425 considered legacy and may be deprecated in the future. You should
426 instead pass them via the ``inputs`` parameter.
427 """
428 + if not self.llm_engine.model_config.embedding_mode:
429 + raise ValueError(
430 + "LLM.encode() is only supported for embedding models (XModel)."
431 + )
432 +
433 if prompt_token_ids is not None or multi_modal_data is not None:
434 inputs = self._convert_v1_inputs(
435 prompts=cast(Optional[Union[str, List[str]]], prompts),