A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[misc][frontend] log all available endpoints (#6195)
Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
youkaichao committed
Jul 7, 2024 at 15:11 UTC
3b08fe2b13ced7fe76abe17c99614dd36e4b4788
2 files changed
+19
vllm/entrypoints/api_server.py
+11
@@ -16,10 +16,13 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse
16
17
from vllm.engine.arg_utils import AsyncEngineArgs
18
from vllm.engine.async_llm_engine import AsyncLLMEngine
19
+from vllm.logger import init_logger
20
from vllm.sampling_params import SamplingParams
21
from vllm.usage.usage_lib import UsageContext
22
from vllm.utils import FlexibleArgumentParser, random_uuid
23
24
+logger = init_logger("vllm.entrypoints.api_server")
25
+
26
TIMEOUT_KEEP_ALIVE = 5 # seconds.
27
app = FastAPI()
28
engine = None
@@ -107,6 +110,14 @@ if __name__ == "__main__":
110
engine_args, usage_context=UsageContext.API_SERVER)
111
112
app.root_path = args.root_path
113
+
114
+ logger.info("Available routes are:")
115
+ for route in app.routes:
116
+ if not hasattr(route, 'methods'):
117
+ continue
118
+ methods = ', '.join(route.methods)
119
+ logger.info("Route: %s, Methods: %s", route.path, methods)
120
+
121
uvicorn.run(app,
122
host=args.host,
123
port=args.port,
vllm/entrypoints/openai/api_server.py
+8
@@ -240,6 +240,14 @@ if __name__ == "__main__":
240
openai_serving_embedding = OpenAIServingEmbedding(engine, model_config,
241
served_model_names)
242
app.root_path = args.root_path
243
+
244
+ logger.info("Available routes are:")
245
+ for route in app.routes:
246
+ if not hasattr(route, 'methods'):
247
+ continue
248
+ methods = ', '.join(route.methods)
249
+ logger.info("Route: %s, Methods: %s", route.path, methods)
250
+
251
uvicorn.run(app,
252
host=args.host,
253
port=args.port,