@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 57f560aa23
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[BugFix] Use args.trust_remote_code (#7121)

Aditya Paliwal committed Aug 5, 2024 at 09:26 UTC 57f560aa23077ed9def5952ab81a65bc080ae234
1 file changed +3 -3
vllm/entrypoints/openai/api_server.py
+3 -3
@@ -60,11 +60,11 @@ logger = init_logger('vllm.entrypoints.openai.api_server')
60 _running_tasks: Set[asyncio.Task] = set()
61
62
63 -def model_is_embedding(model_name: str) -> bool:
63 +def model_is_embedding(model_name: str, trust_remote_code: bool) -> bool:
64 return ModelConfig(model=model_name,
65 tokenizer=model_name,
66 tokenizer_mode="auto",
67 - trust_remote_code=False,
67 + trust_remote_code=trust_remote_code,
68 seed=0,
69 dtype="float16").embedding_mode
70
@@ -97,7 +97,7 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
97
98 # If manually triggered or embedding model, use AsyncLLMEngine in process.
99 # TODO: support embedding model via RPC.
100 - if (model_is_embedding(args.model)
100 + if (model_is_embedding(args.model, args.trust_remote_code)
101 or args.disable_frontend_multiprocessing):
102 async_engine_client = AsyncLLMEngine.from_engine_args(
103 engine_args, usage_context=UsageContext.OPENAI_API_SERVER)