@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ce036244c9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Allow setting fastapi root_path argument (#2341)

Chirag Jain committed Jan 13, 2024 at 00:29 UTC ce036244c9999c90c8cf906bf21cc3f171213333
2 files changed +12
vllm/entrypoints/api_server.py
+6
@@ -74,12 +74,18 @@ if __name__ == "__main__":
74 parser.add_argument("--port", type=int, default=8000)
75 parser.add_argument("--ssl-keyfile", type=str, default=None)
76 parser.add_argument("--ssl-certfile", type=str, default=None)
77 + parser.add_argument(
78 + "--root-path",
79 + type=str,
80 + default=None,
81 + help="FastAPI root_path when app is behind a path based routing proxy")
82 parser = AsyncEngineArgs.add_cli_args(parser)
83 args = parser.parse_args()
84
85 engine_args = AsyncEngineArgs.from_cli_args(args)
86 engine = AsyncLLMEngine.from_engine_args(engine_args)
87
88 + app.root_path = args.root_path
89 uvicorn.run(app,
90 host=args.host,
91 port=args.port,
vllm/entrypoints/openai/api_server.py
+6
@@ -106,6 +106,11 @@ def parse_args():
106 type=str,
107 default=None,
108 help="The file path to the SSL cert file")
109 + parser.add_argument(
110 + "--root-path",
111 + type=str,
112 + default=None,
113 + help="FastAPI root_path when app is behind a path based routing proxy")
114
115 parser = AsyncEngineArgs.add_cli_args(parser)
116 return parser.parse_args()
@@ -766,6 +771,7 @@ if __name__ == "__main__":
771 # Register labels for metrics
772 add_global_metrics_labels(model_name=engine_args.model)
773
774 + app.root_path = args.root_path
775 uvicorn.run(app,
776 host=args.host,
777 port=args.port,