@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 67d115db08
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix][Frontend] Disable embedding API for chat models (#7504)

Co-authored-by: jack <jack@alex>

jack committed Aug 15, 2024 at 00:15 UTC 67d115db08a329bc37f84e37cbc8f7d24c195709
1 file changed +4 -1
vllm/entrypoints/openai/serving_embedding.py
+4 -1
@@ -71,7 +71,7 @@ class OpenAIServingEmbedding(OpenAIServing):
71 lora_modules=None,
72 prompt_adapters=None,
73 request_logger=request_logger)
74 - self._check_embedding_mode(model_config.embedding_mode)
74 + self._enabled = self._check_embedding_mode(model_config.embedding_mode)
75
76 async def create_embedding(
77 self,
@@ -83,6 +83,8 @@ class OpenAIServingEmbedding(OpenAIServing):
83 See https://platform.openai.com/docs/api-reference/embeddings/create
84 for the API specification. This API mimics the OpenAI Embedding API.
85 """
86 + if not self._enabled:
87 + return self.create_error_response("Embedding API disabled")
88 error_check_ret = await self._check_model(request)
89 if error_check_ret is not None:
90 return error_check_ret
@@ -179,3 +181,4 @@ class OpenAIServingEmbedding(OpenAIServing):
181 "embedding_mode is False. Embedding API will not work.")
182 else:
183 logger.info("Activating the server engine with embedding enabled.")
184 + return embedding_mode