A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Frontend] Separate OpenAI Batch Runner usage from API Server (#4851)
Alex Wu committed
May 16, 2024 at 11:42 UTC
5e0391c0406ec225b2c58bc22d5be864a432fe40
2 files changed
+2
-1
vllm/entrypoints/openai/run_batch.py
+1
-1
@@ -101,7 +101,7 @@ async def main(args):
101
102
engine_args = AsyncEngineArgs.from_cli_args(args)
103
engine = AsyncLLMEngine.from_engine_args(
104
- engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
104
+ engine_args, usage_context=UsageContext.OPENAI_BATCH_RUNNER)
105
106
# When using single vLLM without engine_use_ray
107
model_config = await engine.get_model_config()
vllm/usage/usage_lib.py
+1
@@ -90,6 +90,7 @@ class UsageContext(str, Enum):
90
LLM_CLASS = "LLM_CLASS"
91
API_SERVER = "API_SERVER"
92
OPENAI_API_SERVER = "OPENAI_API_SERVER"
93
+ OPENAI_BATCH_RUNNER = "OPENAI_BATCH_RUNNER"
94
ENGINE_CONTEXT = "ENGINE_CONTEXT"
95
96