A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
update benchmark_serving_structured_output to include auto backend (#16438)
Signed-off-by: Chenyaaang <chenyangli@google.com>
Chenyaaang committed
Apr 10, 2025 at 21:25 UTC
d544d141ec3e2f2beda7a1c15bcb5fdf763e7e91
1 file changed
+9
-7
benchmarks/benchmark_serving_structured_output.py
+9
-7
@@ -11,7 +11,7 @@ On the client side, run:
11
--model <your_model> \
12
--dataset json \
13
--structured-output-ratio 1.0 \
14
- --structured-output-backend xgrammar \
14
+ --structured-output-backend auto \
15
--request-rate 10 \
16
--num-prompts 1000
17
@@ -997,12 +997,14 @@ if __name__ == "__main__":
997
type=float,
998
default=1.0,
999
help="Ratio of Structured Outputs requests")
1000
- parser.add_argument(
1001
- "--structured-output-backend",
1002
- type=str,
1003
- choices=["outlines", "lm-format-enforcer", "xgrammar", "guidance"],
1004
- default="xgrammar",
1005
- help="Backend to use for structured outputs")
1000
+ parser.add_argument("--structured-output-backend",
1001
+ type=str,
1002
+ choices=[
1003
+ "outlines", "lm-format-enforcer", "xgrammar",
1004
+ "guidance", "auto"
1005
+ ],
1006
+ default="auto",
1007
+ help="Backend to use for structured outputs")
1008
1009
args = parser.parse_args()
1010
main(args)