@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0e39a33c6d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix][Hardware][AMD][Frontend] add quantization param to embedding checking method (#7513)

Gordon Wong committed Aug 17, 2024 at 00:05 UTC 0e39a33c6d6dd25687db83c410b6210201880bdd
1 file changed +5 -2
vllm/entrypoints/openai/api_server.py
+5 -2
@@ -60,11 +60,13 @@ logger = init_logger('vllm.entrypoints.openai.api_server')
60 _running_tasks: Set[asyncio.Task] = set()
61
62
63 -def model_is_embedding(model_name: str, trust_remote_code: bool) -> bool:
63 +def model_is_embedding(model_name: str, trust_remote_code: bool,
64 + quantization: str) -> bool:
65 return ModelConfig(model=model_name,
66 tokenizer=model_name,
67 tokenizer_mode="auto",
68 trust_remote_code=trust_remote_code,
69 + quantization=quantization,
70 seed=0,
71 dtype="auto").embedding_mode
72
@@ -97,7 +99,8 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
99
100 # If manually triggered or embedding model, use AsyncLLMEngine in process.
101 # TODO: support embedding model via RPC.
100 - if (model_is_embedding(args.model, args.trust_remote_code)
102 + if (model_is_embedding(args.model, args.trust_remote_code,
103 + args.quantization)
104 or args.disable_frontend_multiprocessing):
105 async_engine_client = AsyncLLMEngine.from_engine_args(
106 engine_args, usage_context=UsageContext.OPENAI_API_SERVER)