@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5e0391c040
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Frontend] Separate OpenAI Batch Runner usage from API Server (#4851)

Alex Wu committed May 16, 2024 at 11:42 UTC 5e0391c0406ec225b2c58bc22d5be864a432fe40
2 files changed +2 -1
vllm/entrypoints/openai/run_batch.py
+1 -1
@@ -101,7 +101,7 @@ async def main(args):
101
102 engine_args = AsyncEngineArgs.from_cli_args(args)
103 engine = AsyncLLMEngine.from_engine_args(
104 - engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
104 + engine_args, usage_context=UsageContext.OPENAI_BATCH_RUNNER)
105
106 # When using single vLLM without engine_use_ray
107 model_config = await engine.get_model_config()
vllm/usage/usage_lib.py
+1
@@ -90,6 +90,7 @@ class UsageContext(str, Enum):
90 LLM_CLASS = "LLM_CLASS"
91 API_SERVER = "API_SERVER"
92 OPENAI_API_SERVER = "OPENAI_API_SERVER"
93 + OPENAI_BATCH_RUNNER = "OPENAI_BATCH_RUNNER"
94 ENGINE_CONTEXT = "ENGINE_CONTEXT"
95
96