A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix][Hardware][AMD][Frontend] add quantization param to embedding checking method (#7513)
Gordon Wong committed
Aug 17, 2024 at 00:05 UTC
0e39a33c6d6dd25687db83c410b6210201880bdd
1 file changed
+5
-2
vllm/entrypoints/openai/api_server.py
+5
-2
@@ -60,11 +60,13 @@ logger = init_logger('vllm.entrypoints.openai.api_server')
60
_running_tasks: Set[asyncio.Task] = set()
61
62
63
-def model_is_embedding(model_name: str, trust_remote_code: bool) -> bool:
63
+def model_is_embedding(model_name: str, trust_remote_code: bool,
64
+ quantization: str) -> bool:
65
return ModelConfig(model=model_name,
66
tokenizer=model_name,
67
tokenizer_mode="auto",
68
trust_remote_code=trust_remote_code,
69
+ quantization=quantization,
70
seed=0,
71
dtype="auto").embedding_mode
72
@@ -97,7 +99,8 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
99
100
# If manually triggered or embedding model, use AsyncLLMEngine in process.
101
# TODO: support embedding model via RPC.
100
- if (model_is_embedding(args.model, args.trust_remote_code)
102
+ if (model_is_embedding(args.model, args.trust_remote_code,
103
+ args.quantization)
104
or args.disable_frontend_multiprocessing):
105
async_engine_client = AsyncLLMEngine.from_engine_args(
106
engine_args, usage_context=UsageContext.OPENAI_API_SERVER)