A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add warning to prevent changes to benchmark api server (#2858)
Simon Mo committed
Feb 18, 2024 at 21:36 UTC
86fd8bb0ac9a836e55b5075d8416bd067af9e7b2
1 file changed
+6
vllm/entrypoints/api_server.py
+6
@@ -1,3 +1,9 @@
1
+"""
2
+NOTE: This API server is used only for demonstrating usage of AsyncEngine and simple performance benchmarks.
3
+It is not intended for production use. For production use, we recommend using our OpenAI compatible server.
4
+We are also not going to accept PRs modifying this file, please change `vllm/entrypoints/openai/api_server.py` instead.
5
+"""
6
+
7
import argparse
8
import json
9
from typing import AsyncGenerator