A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Allow setting fastapi root_path argument (#2341)
Chirag Jain committed
Jan 13, 2024 at 00:29 UTC
ce036244c9999c90c8cf906bf21cc3f171213333
2 files changed
+12
vllm/entrypoints/api_server.py
+6
@@ -74,12 +74,18 @@ if __name__ == "__main__":
74
parser.add_argument("--port", type=int, default=8000)
75
parser.add_argument("--ssl-keyfile", type=str, default=None)
76
parser.add_argument("--ssl-certfile", type=str, default=None)
77
+ parser.add_argument(
78
+ "--root-path",
79
+ type=str,
80
+ default=None,
81
+ help="FastAPI root_path when app is behind a path based routing proxy")
82
parser = AsyncEngineArgs.add_cli_args(parser)
83
args = parser.parse_args()
84
85
engine_args = AsyncEngineArgs.from_cli_args(args)
86
engine = AsyncLLMEngine.from_engine_args(engine_args)
87
88
+ app.root_path = args.root_path
89
uvicorn.run(app,
90
host=args.host,
91
port=args.port,
vllm/entrypoints/openai/api_server.py
+6
@@ -106,6 +106,11 @@ def parse_args():
106
type=str,
107
default=None,
108
help="The file path to the SSL cert file")
109
+ parser.add_argument(
110
+ "--root-path",
111
+ type=str,
112
+ default=None,
113
+ help="FastAPI root_path when app is behind a path based routing proxy")
114
115
parser = AsyncEngineArgs.add_cli_args(parser)
116
return parser.parse_args()
@@ -766,6 +771,7 @@ if __name__ == "__main__":
771
# Register labels for metrics
772
add_global_metrics_labels(model_name=engine_args.model)
773
774
+ app.root_path = args.root_path
775
uvicorn.run(app,
776
host=args.host,
777
port=args.port,