A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Update sample prompts in `simple_server.py` (#89)
Woosuk Kwon committed
May 9, 2023 at 16:47 UTC
ce26e57fd3ff4d6f92e5bb48b5e55b23ab5d2171
1 file changed
+3
-3
simple_server.py
+3
-3
@@ -10,9 +10,9 @@ def main(args: argparse.Namespace):
10
server, frontend = init_local_server_and_frontend_with_arguments(args)
11
# Test the following inputs.
12
test_inputs = [
13
- ('Ion Stoica is a', {'n': 4, 'use_beam_search': True, 'temperature': 0.0}),
14
- ('UC Berkeley is', {'n': 3, 'temperature': 0.8, 'top_p': 0.99}),
15
- ('The future of cloud computing is', {}), # Use default parameters.
13
+ ("A robot may not injure a human being", {}), # Use default parameters.
14
+ ("What is the meaning of life?", {"n": 3, "temperature": 0.8, "top_p": 0.99}),
15
+ ("It is only with the heart that one can see rightly", {"n": 4, "use_beam_search": True, "temperature": 0.0}),
16
]
17
while True:
18
if test_inputs: