A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix][Frontend] Disable embedding API for chat models (#7504)
Co-authored-by: jack <jack@alex>
jack committed
Aug 15, 2024 at 00:15 UTC
67d115db08a329bc37f84e37cbc8f7d24c195709
1 file changed
+4
-1
vllm/entrypoints/openai/serving_embedding.py
+4
-1
@@ -71,7 +71,7 @@ class OpenAIServingEmbedding(OpenAIServing):
71
lora_modules=None,
72
prompt_adapters=None,
73
request_logger=request_logger)
74
- self._check_embedding_mode(model_config.embedding_mode)
74
+ self._enabled = self._check_embedding_mode(model_config.embedding_mode)
75
76
async def create_embedding(
77
self,
@@ -83,6 +83,8 @@ class OpenAIServingEmbedding(OpenAIServing):
83
See https://platform.openai.com/docs/api-reference/embeddings/create
84
for the API specification. This API mimics the OpenAI Embedding API.
85
"""
86
+ if not self._enabled:
87
+ return self.create_error_response("Embedding API disabled")
88
error_check_ret = await self._check_model(request)
89
if error_check_ret is not None:
90
return error_check_ret
@@ -179,3 +181,4 @@ class OpenAIServingEmbedding(OpenAIServing):
181
"embedding_mode is False. Embedding API will not work.")
182
else:
183
logger.info("Activating the server engine with embedding enabled.")
184
+ return embedding_mode