@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 837c1968f9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Frontend] Expose revision arg in OpenAI server (#8501)

lewtun committed Sep 16, 2024 at 17:55 UTC 837c1968f9f1fdd9d894b2071d605ca1bdc97942
1 file changed +4 -2
vllm/entrypoints/openai/api_server.py
+4 -2
@@ -69,8 +69,10 @@ _running_tasks: Set[asyncio.Task] = set()
69
70
71 def model_is_embedding(model_name: str, trust_remote_code: bool,
72 - quantization: Optional[str]) -> bool:
72 + quantization: Optional[str],
73 + revision: Optional[str]) -> bool:
74 return ModelConfig(model=model_name,
75 + revision=revision,
76 tokenizer=model_name,
77 tokenizer_mode="auto",
78 trust_remote_code=trust_remote_code,
@@ -130,7 +132,7 @@ async def build_async_engine_client_from_engine_args(
132 # If manually triggered or embedding model, use AsyncLLMEngine in process.
133 # TODO: support embedding model via RPC.
134 if (model_is_embedding(engine_args.model, engine_args.trust_remote_code,
133 - engine_args.quantization)
135 + engine_args.quantization, engine_args.revision)
136 or disable_frontend_multiprocessing):
137 engine_client = AsyncLLMEngine.from_engine_args(
138 engine_args, usage_context=UsageContext.OPENAI_API_SERVER)