A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[bugfix] make args.stream work (#6831)
Wang Ran (汪然) committed
Jul 27, 2024 at 17:07 UTC
a57d75821c6177da75fdebf171d528eef5301961
1 file changed
+4
-1
examples/api_client.py
+4
-1
@@ -31,7 +31,10 @@ def post_http_request(prompt: str,
31
"max_tokens": 16,
32
"stream": stream,
33
}
34
- response = requests.post(api_url, headers=headers, json=pload, stream=True)
34
+ response = requests.post(api_url,
35
+ headers=headers,
36
+ json=pload,
37
+ stream=stream)
38
return response
39
40