@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ce26e57fd3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Update sample prompts in `simple_server.py` (#89)

Woosuk Kwon committed May 9, 2023 at 16:47 UTC ce26e57fd3ff4d6f92e5bb48b5e55b23ab5d2171
1 file changed +3 -3
simple_server.py
+3 -3
@@ -10,9 +10,9 @@ def main(args: argparse.Namespace):
10 server, frontend = init_local_server_and_frontend_with_arguments(args)
11 # Test the following inputs.
12 test_inputs = [
13 - ('Ion Stoica is a', {'n': 4, 'use_beam_search': True, 'temperature': 0.0}),
14 - ('UC Berkeley is', {'n': 3, 'temperature': 0.8, 'top_p': 0.99}),
15 - ('The future of cloud computing is', {}), # Use default parameters.
13 + ("A robot may not injure a human being", {}), # Use default parameters.
14 + ("What is the meaning of life?", {"n": 3, "temperature": 0.8, "top_p": 0.99}),
15 + ("It is only with the heart that one can see rightly", {"n": 4, "use_beam_search": True, "temperature": 0.0}),
16 ]
17 while True:
18 if test_inputs: