A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[BugFix] Use args.trust_remote_code (#7121)
Aditya Paliwal committed
Aug 5, 2024 at 09:26 UTC
57f560aa23077ed9def5952ab81a65bc080ae234
1 file changed
+3
-3
vllm/entrypoints/openai/api_server.py
+3
-3
@@ -60,11 +60,11 @@ logger = init_logger('vllm.entrypoints.openai.api_server')
60
_running_tasks: Set[asyncio.Task] = set()
61
62
63
-def model_is_embedding(model_name: str) -> bool:
63
+def model_is_embedding(model_name: str, trust_remote_code: bool) -> bool:
64
return ModelConfig(model=model_name,
65
tokenizer=model_name,
66
tokenizer_mode="auto",
67
- trust_remote_code=False,
67
+ trust_remote_code=trust_remote_code,
68
seed=0,
69
dtype="float16").embedding_mode
70
@@ -97,7 +97,7 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
97
98
# If manually triggered or embedding model, use AsyncLLMEngine in process.
99
# TODO: support embedding model via RPC.
100
- if (model_is_embedding(args.model)
100
+ if (model_is_embedding(args.model, args.trust_remote_code)
101
or args.disable_frontend_multiprocessing):
102
async_engine_client = AsyncLLMEngine.from_engine_args(
103
engine_args, usage_context=UsageContext.OPENAI_API_SERVER)