A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Misc] V0 fallback for `--enable-prompt-embeds` (#17615)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
May 4, 2025 at 06:59 UTC
46fae69cf04db85a3b187a5fdc061b21e24b9571
2 files changed
+6
-3
vllm/engine/arg_utils.py
+6
@@ -1237,6 +1237,12 @@ class EngineArgs:
1237
recommend_to_remove=False)
1238
return False
1239
1240
+ # No text embedding inputs so far.
1241
+ if self.enable_prompt_embeds:
1242
+ _raise_or_fallback(feature_name="--enable-prompt-embeds",
1243
+ recommend_to_remove=False)
1244
+ return False
1245
+
1246
# Only Fp16 and Bf16 dtypes since we only support FA.
1247
V1_SUPPORTED_DTYPES = [torch.bfloat16, torch.float16]
1248
if model_config.dtype not in V1_SUPPORTED_DTYPES:
vllm/inputs/preprocess.py
-3
@@ -6,7 +6,6 @@ from typing import Any, Optional, Union, cast
6
7
from typing_extensions import assert_never
8
9
-from vllm import envs
9
from vllm.config import ModelConfig
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
@@ -306,8 +305,6 @@ class InputPreprocessor:
305
if not self.model_config.enable_prompt_embeds:
306
raise ValueError("You must set `--enable-prompt-embeds` to input "
307
"`prompt_embeds`.")
309
- if envs.VLLM_USE_V1:
310
- raise ValueError("`prompt_embeds` is only available in V0.")
308
309
prompt_embeds = parsed_content["prompt_embeds"]
310