@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3b08fe2b13
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[misc][frontend] log all available endpoints (#6195)

Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

youkaichao committed Jul 7, 2024 at 15:11 UTC 3b08fe2b13ced7fe76abe17c99614dd36e4b4788
2 files changed +19
vllm/entrypoints/api_server.py
+11
@@ -16,10 +16,13 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse
16
17 from vllm.engine.arg_utils import AsyncEngineArgs
18 from vllm.engine.async_llm_engine import AsyncLLMEngine
19 +from vllm.logger import init_logger
20 from vllm.sampling_params import SamplingParams
21 from vllm.usage.usage_lib import UsageContext
22 from vllm.utils import FlexibleArgumentParser, random_uuid
23
24 +logger = init_logger("vllm.entrypoints.api_server")
25 +
26 TIMEOUT_KEEP_ALIVE = 5 # seconds.
27 app = FastAPI()
28 engine = None
@@ -107,6 +110,14 @@ if __name__ == "__main__":
110 engine_args, usage_context=UsageContext.API_SERVER)
111
112 app.root_path = args.root_path
113 +
114 + logger.info("Available routes are:")
115 + for route in app.routes:
116 + if not hasattr(route, 'methods'):
117 + continue
118 + methods = ', '.join(route.methods)
119 + logger.info("Route: %s, Methods: %s", route.path, methods)
120 +
121 uvicorn.run(app,
122 host=args.host,
123 port=args.port,
vllm/entrypoints/openai/api_server.py
+8
@@ -240,6 +240,14 @@ if __name__ == "__main__":
240 openai_serving_embedding = OpenAIServingEmbedding(engine, model_config,
241 served_model_names)
242 app.root_path = args.root_path
243 +
244 + logger.info("Available routes are:")
245 + for route in app.routes:
246 + if not hasattr(route, 'methods'):
247 + continue
248 + methods = ', '.join(route.methods)
249 + logger.info("Route: %s, Methods: %s", route.path, methods)
250 +
251 uvicorn.run(app,
252 host=args.host,
253 port=args.port,